{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import collections\n",
    "import json\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>v0</th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>v3</th>\n",
       "      <th>v4</th>\n",
       "      <th>v5</th>\n",
       "      <th>v6</th>\n",
       "      <th>v7</th>\n",
       "      <th>v8</th>\n",
       "      <th>...</th>\n",
       "      <th>v290</th>\n",
       "      <th>v291</th>\n",
       "      <th>v292</th>\n",
       "      <th>v293</th>\n",
       "      <th>v294</th>\n",
       "      <th>v295</th>\n",
       "      <th>v296</th>\n",
       "      <th>v297</th>\n",
       "      <th>v298</th>\n",
       "      <th>v299</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>之</td>\n",
       "      <td>-0.386241</td>\n",
       "      <td>-0.200756</td>\n",
       "      <td>0.058861</td>\n",
       "      <td>-0.049590</td>\n",
       "      <td>0.068790</td>\n",
       "      <td>-0.029833</td>\n",
       "      <td>0.106928</td>\n",
       "      <td>0.327273</td>\n",
       "      <td>0.080174</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.103016</td>\n",
       "      <td>0.381482</td>\n",
       "      <td>-0.233765</td>\n",
       "      <td>0.342992</td>\n",
       "      <td>-0.309662</td>\n",
       "      <td>-0.334691</td>\n",
       "      <td>-0.181899</td>\n",
       "      <td>-0.196574</td>\n",
       "      <td>0.561199</td>\n",
       "      <td>-0.010766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>不</td>\n",
       "      <td>0.328010</td>\n",
       "      <td>-0.387920</td>\n",
       "      <td>0.348312</td>\n",
       "      <td>-0.153162</td>\n",
       "      <td>-0.131097</td>\n",
       "      <td>-0.365512</td>\n",
       "      <td>0.083437</td>\n",
       "      <td>0.204496</td>\n",
       "      <td>0.160392</td>\n",
       "      <td>...</td>\n",
       "      <td>0.633649</td>\n",
       "      <td>0.258461</td>\n",
       "      <td>-0.093742</td>\n",
       "      <td>0.580965</td>\n",
       "      <td>-0.103540</td>\n",
       "      <td>-0.715841</td>\n",
       "      <td>-0.179566</td>\n",
       "      <td>-0.147643</td>\n",
       "      <td>0.337213</td>\n",
       "      <td>-0.365105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>以</td>\n",
       "      <td>0.042976</td>\n",
       "      <td>-0.105353</td>\n",
       "      <td>0.189561</td>\n",
       "      <td>0.310238</td>\n",
       "      <td>0.402705</td>\n",
       "      <td>0.055511</td>\n",
       "      <td>-0.015055</td>\n",
       "      <td>0.373181</td>\n",
       "      <td>0.098164</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.164020</td>\n",
       "      <td>0.367616</td>\n",
       "      <td>0.002860</td>\n",
       "      <td>-0.190940</td>\n",
       "      <td>-0.148527</td>\n",
       "      <td>-0.435249</td>\n",
       "      <td>-0.147146</td>\n",
       "      <td>0.100625</td>\n",
       "      <td>0.753493</td>\n",
       "      <td>0.008979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>曰</td>\n",
       "      <td>-0.253626</td>\n",
       "      <td>-0.486539</td>\n",
       "      <td>0.438117</td>\n",
       "      <td>-0.193879</td>\n",
       "      <td>0.134444</td>\n",
       "      <td>0.647642</td>\n",
       "      <td>0.413914</td>\n",
       "      <td>0.227690</td>\n",
       "      <td>0.321753</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.345591</td>\n",
       "      <td>0.074266</td>\n",
       "      <td>0.142636</td>\n",
       "      <td>-0.025664</td>\n",
       "      <td>-0.316415</td>\n",
       "      <td>-0.421364</td>\n",
       "      <td>-0.486560</td>\n",
       "      <td>-0.244505</td>\n",
       "      <td>0.463775</td>\n",
       "      <td>0.182786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>为</td>\n",
       "      <td>-0.034353</td>\n",
       "      <td>-0.075638</td>\n",
       "      <td>0.247427</td>\n",
       "      <td>-0.560503</td>\n",
       "      <td>-0.003090</td>\n",
       "      <td>-0.311814</td>\n",
       "      <td>0.185319</td>\n",
       "      <td>0.283735</td>\n",
       "      <td>0.641233</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043282</td>\n",
       "      <td>-0.097455</td>\n",
       "      <td>-0.298162</td>\n",
       "      <td>-0.289849</td>\n",
       "      <td>-0.052708</td>\n",
       "      <td>-0.595641</td>\n",
       "      <td>-0.160703</td>\n",
       "      <td>0.373042</td>\n",
       "      <td>0.456759</td>\n",
       "      <td>0.597949</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 301 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  word        v0        v1        v2        v3        v4        v5        v6  \\\n",
       "0    之 -0.386241 -0.200756  0.058861 -0.049590  0.068790 -0.029833  0.106928   \n",
       "1    不  0.328010 -0.387920  0.348312 -0.153162 -0.131097 -0.365512  0.083437   \n",
       "2    以  0.042976 -0.105353  0.189561  0.310238  0.402705  0.055511 -0.015055   \n",
       "3    曰 -0.253626 -0.486539  0.438117 -0.193879  0.134444  0.647642  0.413914   \n",
       "4    为 -0.034353 -0.075638  0.247427 -0.560503 -0.003090 -0.311814  0.185319   \n",
       "\n",
       "         v7        v8    ...         v290      v291      v292      v293  \\\n",
       "0  0.327273  0.080174    ...    -0.103016  0.381482 -0.233765  0.342992   \n",
       "1  0.204496  0.160392    ...     0.633649  0.258461 -0.093742  0.580965   \n",
       "2  0.373181  0.098164    ...    -0.164020  0.367616  0.002860 -0.190940   \n",
       "3  0.227690  0.321753    ...    -0.345591  0.074266  0.142636 -0.025664   \n",
       "4  0.283735  0.641233    ...     0.043282 -0.097455 -0.298162 -0.289849   \n",
       "\n",
       "       v294      v295      v296      v297      v298      v299  \n",
       "0 -0.309662 -0.334691 -0.181899 -0.196574  0.561199 -0.010766  \n",
       "1 -0.103540 -0.715841 -0.179566 -0.147643  0.337213 -0.365105  \n",
       "2 -0.148527 -0.435249 -0.147146  0.100625  0.753493  0.008979  \n",
       "3 -0.316415 -0.421364 -0.486560 -0.244505  0.463775  0.182786  \n",
       "4 -0.052708 -0.595641 -0.160703  0.373042  0.456759  0.597949  \n",
       "\n",
       "[5 rows x 301 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2vec_filename = 'sgns.sikuquanshu.bigram'\n",
    "rst = pd.read_csv(word2vec_filename, sep=' ', index_col=False)\n",
    "rst.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#潘\n",
      "#阆\n",
      "#酒\n",
      "#泉\n",
      "#子\n",
      "#（\n",
      "#十\n",
      "#之\n",
      "#一\n",
      "#）\n",
      "#长\n",
      "#忆\n",
      "#钱\n",
      "#塘\n",
      "#不\n"
     ]
    }
   ],
   "source": [
    "filename = \"./data/QuanSongCi.txt\"\n",
    "with open(filename) as f:\n",
    "    context_str = f.read()\n",
    "    \n",
    "nonwords = (',', '.', ' ', '\\n', '，', '。')\n",
    "for word in context_str[0:20]:\n",
    "    if word not in nonwords:\n",
    "        print(\"#\" + word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate=0.001\n",
    "batch_size=16\n",
    "num_steps=32\n",
    "num_words=5000\n",
    "dim_embedding=128\n",
    "rnn_layers=3\n",
    "lstm_size = 600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "global_step = tf.Variable(0, trainable=False, name='self.global_step', dtype=tf.int64)\n",
    "X = tf.placeholder(tf.int32, shape=[batch_size, num_steps], name='input')\n",
    "Y = tf.placeholder(tf.int32, shape=[batch_size, num_steps], name='label')\n",
    "keep_prob = tf.placeholder(tf.float32, name='keep_prob')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_file = 'embedding.npy'\n",
    "embedding = np.load(embedding_file)\n",
    "embed = tf.constant(embedding, name='embedding')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = tf.nn.embedding_lookup(embed, X)\n",
    "y_ = tf.nn.embedding_lookup(embed, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"embedding_lookup:0\", shape=(16, 32, 128), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_cell():\n",
    "    lstm = tf.nn.rnn_cell.LSTMCell(lstm_size, use_peepholes = True)\n",
    "    lstm = tf.nn.rnn_cell.DropoutWrapper(lstm, output_keep_prob = keep_prob, seed = 5)\n",
    "    return lstm\n",
    "    \n",
    "lstm = tf.nn.rnn_cell.MultiRNNCell([ new_cell() for _ in range(rnn_layers)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = lstm.zero_state(batch_size, tf.float32);\n",
    "with tf.variable_scope('softmax'):\n",
    "    softmax_w = tf.get_variable('sw', initializer=tf.truncated_normal((lstm_size, embedding.shape[1])))\n",
    "    softmax_b = tf.get_variable('sb', initializer=tf.truncated_normal((1, embedding.shape[1])))\n",
    "\n",
    "losses = 0\n",
    "for i in range(num_steps):\n",
    "    output, state = lstm(data[:, 1], state)\n",
    "    logits = tf.matmul(output, softmax_w) + softmax_b\n",
    "    loss = tf.square(y_[:, 1] - logits)\n",
    "    losses += loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"add_62:0\", shape=(16, 128), dtype=float32)\n",
      "Tensor(\"concat:0\", shape=(16, 128), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(logits)\n",
    "print(tf.concat(logits, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"multi_rnn_cell/cell_2_31/dropout/mul:0\", shape=(16, 600), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Sum:0' shape=(16,) dtype=float32>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.reduce_sum(losses, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb = np.load('embedding.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 128)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "def test():\n",
    "    for i in range(10):\n",
    "        if i < 8:\n",
    "            yield i\n",
    "        else:\n",
    "            return\n",
    "        \n",
    "for i in test():\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.asarray([1, 1, 1])\n",
    "b = np.asarray([[1, 2, 3],[2, 3, 4]])\n",
    "np.dot(a, b.T).argsort()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 3)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('reverse_dictionary.json', encoding='utf-8') as inf:\n",
    "    reverse_dictionary = json.load(inf, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'亥'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reverse_dictionary['2276']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "shapes (300,) and (128,5000) not aligned: 300 (dim 0) != 128 (dim 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-767c6fc28d65>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mtest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0memb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margsort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: shapes (300,) and (128,5000) not aligned: 300 (dim 0) != 128 (dim 0)"
     ]
    }
   ],
   "source": [
    "test = []\n",
    "for i in range(300):\n",
    "    test.append(i)\n",
    "    \n",
    "np.dot(np.asarray(test), emb.T).argsort()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb[0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
